<?php
// +----------------------------------------------------------------------
// | ThinkCMF [ WE CAN DO IT MORE SIMPLE ]
// +----------------------------------------------------------------------
// | Copyright (c) 2013-2018 http://www.thinkcmf.com All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: guosheng1987@126.com 
// +----------------------------------------------------------------------
namespace plugins\crawler\service; 

use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Pool;
use GuzzleHttp\Promise;
use GuzzleHttp\Cookie\CookieJar;

use phpQuery;
//use QL\QueryList;

use plugins\crawler\model\CrawlerSiteModel;
use plugins\crawler\model\CrawlerUrlModel;
use plugins\crawler\model\CrawlerRuleModel;
use plugins\crawler\model\CrawlerCollectionModel;

use app\portal\model\PortalPostModel;
use app\portal\model\PortalCategoryModel;

use think\Db;

class CrawlerWeihongbaoService extends CrawlerService
{   
    //GuzzleHttp Client 
    public $client;
    //http headers
    public $headers;
    // current crawler site object
    public $site;
    // current crawler url object
    public $url;

    public $collections;
    /**
     * 构造函数
     *
     */
    public function __construct()
    {
        $this->setClient();

        $this->setHeaders();    
    }   

    /**
     * 设置请求类客户端
     * @access public
     * @return void
     */
    public function setClient()
    {
        $this->client = new Client([
            // Base URI is used with relative requests
            'base_uri' =>  'https://www.weihongbao.net',
            // You can set any number of default request options.
            'timeout'  => 30.0,
            'verify'   => false,
            'delay'    => 3000,
        ]);
        
    }

    /**
     * 设置请求头信息
     * @access public
     * @return void
     */
    public function setHeaders()
    {
        $headers = [
            'User-Agent'      => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
            'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding' => 'gzip, deflate',
            'Accept-Language' => 'zh-CN,zh;q=0.9',
            'Cache-Control'   => 'max-age=0',
            'Connection'      => 'keep-alive',
            'Host'            => 'www.weihongbao.net',
            'Referer'         => 'https://www.baidu.com/link?url=8GQpytDvn8MGH4ZZWv7AjF3GP_4lucAZN-gtJIkgJ1ZmWTGmPZbDHX7GerHntauc7RBpFtAs7Y2Zfogn3Y1BPa&wd=&eqid=91bec5ad0000c805000000055af3f274', 
        ];
        
        $cookies = [
            'Hm_lvt_9c9e4e5987185e705fdd349134ab467b='. time(),
            'Hm_lpvt_9c9e4e5987185e705fdd349134ab467b='. time(),
            'yd_cookie=7e117d78-eeb6-4f93b0f8b6dddba90f73293dad9c891ebcea',
            'PHPSESSID=k801hdqcdad9p0j4hgntoh2d54',
            'YFIndex__curUrl_=czoxOiIvIjs%3D',
            'UM_distinctid=16348d8fce6319-0f9832310b7c04-b7a103e-1fa400-16348d8fce7bd5',
            'CNZZDATA1263340661=1289820938-'.time().'-%7C'.time(),
        ];

        $headers['Cookie'] = implode(' ', $cookies);

        $this->headers = $headers;
    }

    /**
     * get collection path and Thumb img from list page
     * @access public
     * @param  String $contents
     * @return Array $collection
     */
    public function parseCollectionPathAndThumbsByContents($contents)
    {
        $collections = [];

        if($contents)
        {
            phpQuery::newDocumentHTML($contents);
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            foreach(pq('div.qun-content ul.w1000 div.dt a') as $a)
            {
                $item['paths']  = pq($a)->attr('href');
                $item['thumbs'] = preg_replace('|\/Uploads\/([\s\S]+?)\/Uploads\/|','/Uploads/', pq($a)->find('img')->attr('src'));

                $item['thumbs'] = str_replace('http://www.weihongbao.net','', $item['thumbs']);

                $collections[] = $item;
            }

            phpQuery::$documents = null;
        
            //$collectionPaths = $qlData->all();
        }

    
        return $collections;
    }

    /**
     * parse title from article page html
     * @access public
     * @param  String $contents
     * @return String $title
     */
    public function parseCollectionTitleByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents);
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $title = pq('div.c-content.command div.box1 h4')->text();

            phpQuery::$documents = null;
        }

        return $title;
    }

    /**
     * parse content from article page html
     * @access public
     * @param  String $contents
     * @return String $maintext
     */
    public function parseCollectionMainTextByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents);
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            $maintext = '';
            foreach(pq('div.fRight.b2 p') as $p)
            {
                $maintext  .= '<p>'. pq($p)->html() .'</p>';
            }

            phpQuery::$documents = null;

        }

        return $maintext;
    }
    
    
    /**
     * parse content image from article page html
     * @access public
     * @param  String $contents
     * @return String $imagePath
     */
    public function parseCollectionContentImageByContents($contents)
    {
        if($contents)
        {
            phpQuery::newDocumentHTML($contents);
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $imagePath = pq('div.fLeft.b1 p img')->attr('src');

            phpQuery::$documents = null;

        }


        return $imagePath;
    }

    /**
     * generate portal data
     * @access private
     * @param  String $categoryName
     * @param  Array $collection 
     * @return Array data
     */
    private function makePortalData($categoryName, $collection)
    {

        $data = [];

        $data['title'] = $collection['title'];
        $data['excerpt'] = mb_substr(strip_tags($collection['content']), 0, 50);
        $data['thumbnail'] = $collection['thumbs'];

        $times = preg_match('/<span[^>]+>([\s\S]+)<\/span>/', $collection['content'], $matches);
        $wechatNo = $matches[1];
        $image = '<br><img src="'.$collection['qrcode'].'" style="box-sizing: border-box; border: 0px; vertical-align: middle; max-width: 100%; height: auto !important;"><br/>';
        $bootomtext = '分享喵(www.fenxiangmiao.com)，将持续不断的为网友们分享'. $categoryName .'，请分享给你们的朋友们哦。群主微信号是：<strong>'. $wechatNo .'</strong>，本群关注人数 <span style="color:#FF4309;">'.rand(100, 100000).'</span><br>';
        
        $data['content'] = $collection['content']. $image. $bootomtext;

        return $data;
    } 

    /**
     * save collection to db
     * @access public
     * @param  Int $index 
     * @return void
     */
    public function saveCrawlerData($index)
    {
        //先删除关联表，防止重复
        $this->deletePortal($this->collections[$index]['collection_id']);
        
        $categoryId = $this->url->crlrule->category_id;
        
        $categoryName = $this->getPortalCategoryNameById($categoryId);

        $portal = $this->makePortalData($categoryName, $this->collections[$index]);

        $postId = $this->addPortalAndTag($categoryId, $portal['title'], $portal['excerpt'] ,$portal['content'], $portal['thumbnail']);

        $this->addCrawlerCollection($postId, $this->collections[$index]);
    }
    
    /**
     * request list url and aget 
     * @access public
     * @return string
     */
    public function reuqestListPage()
    {
        $response = $this->client->request('GET', $this->url->listurl, [
            'allow_redirects' => false,
            'headers'         => $this->headers,
        ]);
        
        $contents = $response->getBody()->getContents();

        return $contents;

    }

    /**
     * request Qrcode image url in article page
     * @access public
     * @param  String $imagePath 
     * @return string
     */
    public function reuqestQrcodeImage($imagePath)
    {
        
        $response = $this->client->request('GET', $this->site->siteurl . $imagePath, [
            'allow_redirects' => false,
            //'headers'         => $this->headers,
        ]);
        
        
        $localFile = $this->saveLocalFile($response->getBody()->getContents(), $imagePath);
        
        return $localFile['url'];
    }



    /**
     * request Thumbs url and save it to local disk
     * @access public
     * @return string
     */
    public function reuqestListThumbs()
    {
        foreach($this->collections as $collection)
        {
            // Initiate each request but do not block
            $promises[] = $this->client->getAsync($collection['thumbs']);
        }

        // Wait on all of the requests to complete.
        $results = Promise\unwrap($promises);

        foreach($results as $i => $result)
        {
            $localFile = $this->saveLocalFile($result->getBody()->getContents(), $this->collections[$i]['thumbs']);

            $this->collections[$i]['thumbs'] = $localFile['url'];

            $this->output->writeln("thumbs: ". $this->collections[$i]['thumbs'] .' request success!');
        }
        
    }
    

    public function requestCollectionPages()
    {

        $requests = function () {
            foreach ($this->collections as $i => $collection) {
                yield new Request('GET', $this->site->siteurl . $collection['paths'], $this->headers);                                
            }
        };

        $pool = new Pool($this->client, $requests(), [
            'concurrency' => 3,
            'fulfilled'   => function ($response, $index) {
                
                $contents = $response->getBody()->getContents();

                $title = $this->parseCollectionTitleByContents($contents);

                $maintext = $this->parseCollectionMainTextByContents($contents);
               
                $imagePath = $this->parseCollectionContentImageByContents($contents);

                $qrcodePath = $this->reuqestQrcodeImage($imagePath);

                $matchtimes = preg_match('/\d+/', $this->collections[$index]['paths'], $matches);

                $this->collections[$index]['title']           = $title;
                $this->collections[$index]['content']         = $maintext;
                $this->collections[$index]['qrcode']          = $qrcodePath;
                $this->collections[$index]['collection_id']   = $matches[0];

                $this->saveCrawlerData($index);
            
                $this->output->writeln("page url ".$this->collections[$index]['paths'] .' collect data success!');         
            },
            'rejected'   => function ($reason, $index) {
                
                //var_dump($reason);

                // this is delivered each failed request
                $this->output->writeln("page url ".$this->collections[$index]['paths'] .' request fail!');

            },
        ]);

        // Initiate the transfers and create a promise
        $promise = $pool->promise();

        // Force the pool of requests to complete.
        $promise->wait();
        
    }
 
 
    /**
     * start crawler job
     * 
     * @return array
     */
    public function start()
    {
        $this->site = CrawlerSiteModel::find(2);

        $urls = CrawlerUrlModel::where('crawler_site_id', $this->site->id)->select();
        
        foreach($urls as $url)
        {   
            $this->url = $url;
           
            $contents = $this->reuqestListPage();

            $this->output->writeln("list url: ". $this->url->listurl.' request success!');
            
            $this->collections = $this->parseCollectionPathAndThumbsByContents($contents);
            
            $this->collections = array_slice($this->collections, 0, 7);
                        
            $this->reuqestListThumbs(); 
           
            $this->requestCollectionPages();

            $this->output->writeln("list url ".$this->url->listurl.' all done!');
            
        }
        $this->output->writeln("Command executed success!");
        
    }
    
}